#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Project: spd-sxmcc
'''
@author: lyndon
@time Created on 2018/9/10 17:59
@desc
'''
import re

from pyspider.libs.base_handler import *
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base


class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes=24 * 60)
    def on_start(self):
        self.crawl('http://www.imdb.com/search/title?count=100&title_type=feature,tv_series,tv_movie&ref_=nv_ch_mm_1',
                   callback=self.index_page, validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('a[href^="http"]').items():
            if re.match("https://www.imdb.com/title/tt\d+/\?ref_=adv_li_i", each.attr.href):
                self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False)
            self.crawl([x.attr.href for x in response.doc('.next-page').items()], callback=self.index_page,
                       validate_cert=False)

    @config(priority=2)
    def detail_page(self, response):
        cretors = ""
        try:
            participants = [x.text() for x in response.doc('.credit_summary_item').items()]
            creators = [e.replace('\n', '').split(':')[1] for e in participants if (
                e.replace('\n', '').split(':')[0].__eq__('Creators')
                or e.replace('\n', '').split(':')[0].__eq__('Creator')
                or e.replace('\n', '').split(':')[0].__eq__('Director'))]
            cretors = creators[0].split(',')
        except Exception:
            pass

        ratingWrapper = ""
        score = "0"
        try:
            ratingWrapper = response.doc('.ratings_wrapper').text()
            scoreFormula = ratingWrapper.split('\n')[0]
            score = scoreFormula.split('/')[0]
        except Exception:
            pass

        title = ""
        h1Header = []
        tag = []
        try:
            h1Header = response.doc('h1')
            uniH1 = h1Header('h1').text()
            title = unicode(uniH1).encode("utf-8")
            for ea in h1Header.items('a'):
                tag.append(ea.text())
            #title = str(response.doc('h1').text())
        except Exception:
            pass

        session = sessionmaker(bind=engine)
        session = session()
        imdbmoveins = ImdbMove(director=str(cretors),
                               rating=str(score),
                               title=title,
                               url=str(response.url))
        session.add(imdbmoveins)
        session.commit()
        return {
            "tag": tag,
            "url": response.url,
            "title": title,
            "rating": ratingWrapper,
            "director": cretors,
            "score": score,
        }


'''
@author: lyndon
@time Created on 2018/9/10 16:24
@desc
'''

engine = create_engine('mysql+mysqldb://root:newpasswd@192.168.100.162:3306/resultdb?charset=utf8')
Base = declarative_base()
print(engine)


class ImdbMove(Base):
    __tablename__ = 'imdb_moves'

    id = Column(Integer, primary_key=True)
    director = Column(String(64))
    rating = Column(String(64))
    title = Column(String(64))
    url = Column(String(1024))

    def __repr__(self):
        """  这个方法似java的 toString方法 """
        return '%s(%r)' % (self.__class__.__name__, self.url)


if __name__ == '__main__':
    Base.metadata.create_all(engine)
